// Copyright 2008 Steve Spiller. All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without 
// modification, are permitted provided that the following conditions are met:
// 
//    1. Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//    2. Redistributions in binary form must reproduce the above copyright 
//       notice, this list of conditions and the following disclaimer in the 
//       documentation and/or other materials provided with the distribution.
//       
// THIS SOFTWARE IS PROVIDED BY THE KAOS PROJECT ``AS IS'' AND ANY EXPRESS 
// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 
// NO EVENT SHALL THE PROJECT OWNERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "LpcScanner.h"

const char *CLpcScanner::tokenNames[] =
{
    "None", "Identifier", "Literal", "Array", "Break", "Buffer", 
    "Case", "Catch", "Class", "Continue", "Default", "Do", "Efun", "Else",
    "Float", "For", "ForEach", "Function", "If", "In", "Inherit", "Int",
    "Mapping", "Mixed", "New", "NoMask", "NoSave", "Object", "Private", 
    "Protected", "Public", "Ref", "Return", "Static", "String", "Switch", 
    "VarArgs", "Void", "While", "AndAnd", "OrOr", "EqualEqual", "NotEqual",
    "LessThan", "GreaterThan", "RightShift", "LeftShift", "PlusPlus", 
    "MinusMinus", "Arrow", "Scope", "RightShiftEqual", "LeftShiftEqual", 
    "PlusEqual", "MinusEqual", "TimesEqual", "DivideEqual", "ModulusEqual",
    "AndEqual", "CaratEqual", "OrEqual", "TildeEqual", "GreaterThanEqual",
    "LessThanEqual", "OpenMapping", "CloseMapping", "OpenFunction", 
    "CloseFunction", "OpenArray", "CloseArray", "Ellipsis",
    "DotDot", "Carat", "Pipe", "Question", "Dollar", "SemiColon", "OpenBrace",
    "CloseBrace", "Comma", "Colon", "Assign", "OpenParen", "CloseParen",
    "OpenBracket", "CloseBracket", "Period", "Ampersand", "Bang", "Tilde",
    "Minus", "Plus", "Star", "Slash", "Percent", "EndOfFile"
};

// ***************************************************************************
// Function Name: CLpcScanner
//
// Purpose: Class Constructor
//
// Access: Public
//
// Arguments:
//    std::string file - Path of file to scan
//
// Return Values:
//    None
//
// Description:
//    Initializes the scanner class
CLpcScanner::CLpcScanner(std::string file)
    : m_fileName(file),
      m_outputStream(0),
      m_nestLevel(0),
      m_inArray(false)
{
    PopulateReservedWords();
}

// ***************************************************************************
// Function Name: ~CLpcScanner
//
// Purpose: Class Destructor
//
// Access: Public
//
// Arguments:
//    None
//
// Return Values:
//    None
//
// Description:
//    Cleans up the scanner class
CLpcScanner::~CLpcScanner()
{
    while(!m_fileStack.empty())
    {
        CLpcFile *pFile = m_fileStack.top();

        delete pFile;

        m_fileStack.pop();
    }
}

// ***************************************************************************
// Function Name: PopulateReservedWords
//
// Purpose: Initialize reserved words table
//
// Access: Private
//
// Arguments:
//    None
//
// Return Values:
//    None
//
// Description:
//    Fills the reserved word map with the tokens for each reserved
//    word that the scanner recognizes.
void
CLpcScanner::PopulateReservedWords()
{
    m_reservedWords[std::string("array")]     = CLpcScanner::Array;
    m_reservedWords[std::string("break")]     = CLpcScanner::Break;
    m_reservedWords[std::string("buffer")]    = CLpcScanner::Buffer;
    m_reservedWords[std::string("case")]      = CLpcScanner::Case;
    m_reservedWords[std::string("catch")]     = CLpcScanner::Catch;
    m_reservedWords[std::string("class")]     = CLpcScanner::Class;
    m_reservedWords[std::string("continue")]  = CLpcScanner::Continue;
    m_reservedWords[std::string("default")]   = CLpcScanner::Default;
    m_reservedWords[std::string("do")]        = CLpcScanner::Do;
    m_reservedWords[std::string("efun")]      = CLpcScanner::Efun;
    m_reservedWords[std::string("else")]      = CLpcScanner::Else;
    m_reservedWords[std::string("float")]     = CLpcScanner::Float;
    m_reservedWords[std::string("for")]       = CLpcScanner::For;
    m_reservedWords[std::string("foreach")]   = CLpcScanner::ForEach;
    m_reservedWords[std::string("function")]  = CLpcScanner::Function;
    m_reservedWords[std::string("if")]        = CLpcScanner::If;
    m_reservedWords[std::string("in")]        = CLpcScanner::In;
    m_reservedWords[std::string("inherit")]   = CLpcScanner::Inherit;
    m_reservedWords[std::string("int")]       = CLpcScanner::Int;
    m_reservedWords[std::string("mapping")]   = CLpcScanner::Mapping;
    m_reservedWords[std::string("mixed")]     = CLpcScanner::Mixed;
    m_reservedWords[std::string("new")]       = CLpcScanner::New;
    m_reservedWords[std::string("nomask")]    = CLpcScanner::NoMask;
    m_reservedWords[std::string("nosave")]    = CLpcScanner::NoSave;
    m_reservedWords[std::string("object")]    = CLpcScanner::Object;
    m_reservedWords[std::string("private")]   = CLpcScanner::Private;
    m_reservedWords[std::string("protected")] = CLpcScanner::Protected;
    m_reservedWords[std::string("public")]    = CLpcScanner::Public;
    m_reservedWords[std::string("ref")]       = CLpcScanner::Ref;
    m_reservedWords[std::string("return")]    = CLpcScanner::Return;
    m_reservedWords[std::string("static")]    = CLpcScanner::Static;
    m_reservedWords[std::string("string")]    = CLpcScanner::String;
    m_reservedWords[std::string("switch")]    = CLpcScanner::Switch;
    m_reservedWords[std::string("varargs")]   = CLpcScanner::VarArgs;
    m_reservedWords[std::string("void")]      = CLpcScanner::Void;
    m_reservedWords[std::string("while")]     = CLpcScanner::While;
}

// ***************************************************************************
// Function Name: GetReservedWordToken
//
// Purpose: Retrieve a reserved word's token value
//
// Access: Private
//
// Arguments:
//    std::string word - The reserved word to find
//
// Return Values:
//    LpcToken - The token value of the word if found or LpcToken::None if not
//
// Description:
//    Looks in the reserved word table for the given identifier and returns
//    the token value if it is found.
CLpcScanner::LpcToken
CLpcScanner::GetReservedWordToken(std::string word)
{
    std::map<std::string, CLpcScanner::LpcToken>::iterator it;

    it = m_reservedWords.find(word);
    
    if(it == m_reservedWords.end())
    {
        return CLpcScanner::None;
    }
    
    return (*it).second;    
}

// ***************************************************************************
// Function Name: GetToken
//
// Purpose: Retrieve the next token in the stream
//
// Access: Public
//
// Arguments:
//    None
//
// Return Values:
//    CLpcToken * - Pointer to a CLpcToken class
//
// Description:
//    Retrieves the next token value from the input stream.  A CLpcToken
//    object is contructed and must be freed by the caller.  The special
//    token LpcToken::EndOfFile is returned when there are no more tokens
//    in the input stream.  If unparsable data is found then an exception
//    of type ScannerException will be thrown.
CLpcToken *
CLpcScanner::GetToken()
{
    if(m_fileStack.empty())
    {
        CLpcFile *pFile = new CLpcFile(m_fileName);

        pFile->SetOutputStream(*m_outputStream);
        pFile->Load();

        m_fileStack.push(pFile);
    }

    CHAR current = File()->Current();

    // Read in the next non-whitespace character
    while(IS_WHITESPACE(current) && File()->EoF() == false)
    {
        current = File()->ReadNext();
    }

    if(current == '#')
    {
        // Pre-processor directives don't cause a token to be returned
        // themselves, they modify the input stream and create/remove
        // macro definitions
        HandleDirective();

        // The current character is likely to have changed so we need
        // to reset it
        current = File()->Current();
    }

    // Check for EoF
    if(File()->EoF() == true)
    {
        CLpcFile *pFile = m_fileStack.top();

        m_fileStack.pop();

        delete pFile;

        if(m_fileStack.empty())
        {
            return new CLpcToken(CLpcScanner::EndOfFile, std::string("eof"));
        }

        // We've popped up a file, so read in the next non-ws character
        current = File()->ReadNext();

        while(IS_WHITESPACE(current) && File()->EoF() == false)
        {
            current = File()->ReadNext();
        }
    }

    // Process the token
    if(IS_LETTER(current) || current == '_')
    {
        return ProcessIdentifier();
    }
    else if(IS_DIGIT(current))
    {
        return ProcessNumber();
    }
    else if(current == '\"')
    {
        // Set a flag so that we won't try to skip characters
        // inside of a string literal
        File()->SetInString(true);

        return ProcessString();
    }
    else if(current == '\'')
    {
        return ProcessCharacter();
    }
    else
    {
        return ProcessSpecial();
    }

    throw ScannerException("Unknown character found in input", 
                           File()->LineNumber(), 
                           File()->Position());
}

// ***************************************************************************
// Function Name: ProcessIdentifier
//
// Purpose: Scan an identifier
//
// Access: Private
//
// Arguments:
//    None
//
// Return Values:
//    CLpcToken * - Pointer to a CLpcToken class
//
// Description:
//    Scans the input from the current character looking for an identifier.
//    When scanning is complete, it checks the reserved word table and
//    returns the proper token value for reserved words, otherwise the
//    generic LpcToken::Identifier value.
//
// Rule:
//    Identifiers can start with a letter or an underscore and contain any
//    number of letters, numbers and underscores after the first character.
CLpcToken *
CLpcScanner::ProcessIdentifier()
{
    std::string ident;
    LpcToken    token;
    CHAR        current = File()->Current();
    
    //
    // Keep pulling off characters until we hit a non-number/alpha/underscore
    //
    while(File()->EoF() == false  && 
         !IS_WHITESPACE(current)  &&
         (IS_DIGIT(current)       || 
          IS_LETTER(current)      ||
          current == '_'))
    {
        ident.append(1, current);
        
        current = File()->ReadNext();
    }
    
    // Check to see if it's a reserved word
    token = GetReservedWordToken(ident);
    
    // If not, just use the Identifier token
    if(token == CLpcScanner::None)
    {
        token = CLpcScanner::Identifier;
    }
    
    if(m_outputStream)
    {
        *m_outputStream << "     >> " << std::setiosflags(std::ios::left);
        *m_outputStream << std::setw(16) << tokenNames[token] << ident << std::endl;
    }
    
    return new CLpcToken(token, ident);
}

// ***************************************************************************
// Function Name: ProcessString
//
// Purpose: Scan a string literal
//
// Access: Private
//
// Arguments:
//    None
//
// Return Values:
//    CLpcToken * - Pointer to a CLpcToken class
//
// Description:
//    Called when a " character was found and scans until a closing " is
//    found, returning the value as a string literal.  If an EoF condition
//    occurs during scanning, an exception of type ScannerException will
//    be thrown.
//
// Rule:
//    Look for a non-excaped " character and return the value between the "s
CLpcToken *
CLpcScanner::ProcessString()
{
    std::string str     = "";
    CHAR        current = File()->Current();
        
    // The current character is the ", so skip it
    current = File()->ReadNext();
        
    while(File()->EoF() == false)
    {
        if(current == '\"')
        {
            break;
        }
        
        if(current == '\\')
        {
            if(File()->PeekNext() == '\"')
            {
                // Escaped quotes become just quotes
                current = File()->ReadNext();
            }
        }
        
        str.append(1, current);
        
        current = File()->ReadNext();
    }

    if(File()->EoF() == true)
    {
        throw ScannerException("End of file in string", 
                               File()->LineNumber(), 
                               File()->Position());
    }
    
    // Reset the 'in string' flag
    File()->SetInString(false);

    // The current character is the closing ", so skip past it
    File()->ReadNext();
    
    if(m_outputStream)
    {
        *m_outputStream << "     >> " << std::setiosflags(std::ios::left);
        *m_outputStream << std::setw(16) << tokenNames[CLpcScanner::Literal] << "\"";
        *m_outputStream << str << "\"" << std::endl;
    }

    return new CLpcToken(CLpcScanner::Literal, str, CLpcScanner::StringLiteral, (char *) str.c_str());
}

// ***************************************************************************
// Function Name: ProcessCharacter
//
// Purpose: Scan a character literal
//
// Access: Private
//
// Arguments:
//    None
//
// Return Values:
//    CLpcToken * - Pointer to a CLpcToken class
//
// Description:
//    Called when a ' character was found and scans until a closing ' is
//    found, returning the value as a numeric literal.  If an EoF condition
//    occurs during scanning, or the rule is broken, an exception of type 
//    ScannerException will be thrown.
//
// Rule:
//    Look for a single character between ' marks, e.g. '#', '\t', '\0', '\x0f'
CLpcToken *
CLpcScanner::ProcessCharacter()
{
    INT         value   = 0;
    std::string str     = "";
    CHAR        current = File()->ReadNext();

    if(File()->EoF() == true)
    {
        throw ScannerException("End of file in character constant", 
                               File()->LineNumber(), 
                               File()->Position());
    }

    // If the next character is a ' then it's just a single character
    if(File()->PeekNext() == '\'')
    {
        value = current;
        str.append(1, current);

        current = File()->ReadNext(); // Read the ending '
    }
    else if(current == '\\')
    {
        //
        // Process an escaped character
        //
        current = File()->ReadNext(); // Read the first part of the escape sequesnce

        switch(current)
        {
            case 'n': str.append(1, '\n'); value = '\n'; current = File()->ReadNext(); break;
            case 't': str.append(1, '\t'); value = '\t'; current = File()->ReadNext(); break;
            case 'r': str.append(1, '\r'); value = '\r'; current = File()->ReadNext(); break;
            case 'b': str.append(1, '\b'); value = '\b'; current = File()->ReadNext(); break;
            case 'a': str.append(1, '\x07'); value = '\x07'; current = File()->ReadNext(); break;
            case 'e': str.append(1, '\x1b'); value = '\x1b'; current = File()->ReadNext(); break;
            case '\'': str.append(1, '\''); value = '\''; current = File()->ReadNext(); break;
            case '"': str.append(1, '"'); value = '"'; current = File()->ReadNext(); break;
            case '\\': str.append(1, '\\'); value = '\\'; current = File()->ReadNext(); break;
            case '0': case '1': case '2': case '3': case '4':
            case '5': case '6': case '7': case '8': case '9':
                {
                    //
                    // Process escaped octal number
                    //
                    std::string number;

                    while(File()->EoF() == false && IS_DIGIT(current))
                    {
                        number.append(1, current);
                        current = File()->ReadNext();
                    }

                    if(File()->EoF() == true)
                    {
                        throw ScannerException("End of file in character constant", 
                                               File()->LineNumber(), 
                                               File()->Position());
                    }

                    std::stringstream temp(number);
                    INT               val;

                    temp >> std::oct >> val;

                    if(val > 255)
                    {
                        throw ScannerException("Character constant too big", 
                                               File()->LineNumber(), 
                                               File()->Position());
                    }

                    value = val;
                    str.append(1, val);
                }
                break;

            case 'x':
                {
                    //
                    // Process escaped hex value
                    //
                    std::string number("0x");

                    current = File()->ReadNext();

                    while(File()->EoF() == false && IS_HEXDIGIT(current))
                    {
                        number.append(1, current);
                        current = File()->ReadNext();
                    }

                    if(File()->EoF() == true)
                    {
                        throw ScannerException("End of file in character constant", 
                                               File()->LineNumber(), 
                                               File()->Position());
                    }

                    std::stringstream temp(number);
                    INT               val;

                    temp >> std::hex >> val;

                    if(val > 255)
                    {
                        throw ScannerException("Character constant too big", 
                                               File()->LineNumber(), 
                                               File()->Position());
                    }
                    
                    value = val;
                    str.append(1, val);
                }
                break;

            default: throw ScannerException("Unknown escape sequence", 
                                            File()->LineNumber(), 
                                            File()->Position());
        }
    }
    else
    {
        throw ScannerException("Invalid character constant", 
                               File()->LineNumber(), 
                               File()->Position());
    }

    File()->ReadNext(); // Read the character after the closing '

    if(m_outputStream)
    {
        *m_outputStream << "     >> " << std::setiosflags(std::ios::left);
        *m_outputStream << std::setw(16) << tokenNames[CLpcScanner::Literal];
        *m_outputStream << (int) value << " (char '";
        
        if(value < 32 || value > 126)
        {
            *m_outputStream << "\\x" << std::setiosflags(std::ios::right) << std::setw(2);
            *m_outputStream << std::setfill('0') << std::hex << (int) value << "')";
        }
        else
        {
            *m_outputStream << str << "')";
        }

        *m_outputStream << std::setfill(' ') << std::dec << std::resetiosflags(std::ios::right) << std::endl;
    }

    return new CLpcToken(CLpcScanner::Literal, str, CLpcScanner::IntegerLiteral, value);
}

// ***************************************************************************
// Function Name: ProcessNumber
//
// Purpose: Scan a numeric literal
//
// Access: Private
//
// Arguments:
//    None
//
// Return Values:
//    CLpcToken * - Pointer to a CLpcToken class
//
// Description:
//    Scans and converts integer and floating pointer numbers into literals.
//
// Rule:
//    Scan digits until a non-digit character.  If a '.' is found, keep it
//    and note that the number is a float instead of an int.
CLpcToken *
CLpcScanner::ProcessNumber()
{
    std::string number;
    CLpcToken * token   = 0;
    bool        isFloat = false;
    CHAR        current = File()->Current();
            
    while(File()->EoF() == false  && 
         !IS_WHITESPACE(current)  &&
         (IS_DIGIT(current)       || 
          current == '.'))
    {
        number.append(1, current);
        
        if(current == '.')
        {
            if(File()->PeekNext() == '.')
            {
                // Need to handle the case of array ranges e.g. [1..2]
                // Exiting the loop here
                break;
            }
            
            isFloat = true;
        }
        
        current = File()->ReadNext();
    }

    //
    // Convert from the string representation into a native data type
    //
    std::stringstream temp(number);
    
    if(isFloat == true)
    {
        FLOAT value;
        
        temp >> value;
        
        token = new CLpcToken(CLpcScanner::Literal, number, CLpcScanner::RealLiteral, value);
    }
    else
    {
        INT value;
        
        temp >> value;
        
        token = new CLpcToken(CLpcScanner::Literal, number, CLpcScanner::IntegerLiteral, value);
    }
    
    if(m_outputStream)
    {
        *m_outputStream << "     >> " << std::setiosflags(std::ios::left);
        *m_outputStream << std::setw(16) << tokenNames[CLpcScanner::Literal];
        
        if(isFloat == true)
        {
            *m_outputStream << token->GetRealLiteral() << " (real)" << std::endl;
        }
        else
        {
            *m_outputStream << token->GetIntLiteral() << " (integer)" << std::endl;
        }
    }

    return token;
}

// ***************************************************************************
// Function Name: ProcessSpecial
//
// Purpose: Scan a character literal
//
// Access: Private
//
// Arguments:
//    None
//
// Return Values:
//    CLpcToken * - Pointer to a CLpcToken class
//
// Description:
//    This is the catch-all processing routine.  It scans for all the tokens that
//    are made up of non-alphanumeric values
CLpcToken *
CLpcScanner::ProcessSpecial()
{
    LpcToken    token   = CLpcScanner::None;
    std::string meme    = "";
    
    switch(File()->Current())
    {
        case '-':
            switch(File()->PeekNext())
            {
                case '-': token = CLpcScanner::MinusMinus; meme = "--"; File()->ReadNext(); break;
                case '=': token = CLpcScanner::MinusEqual; meme = "-="; File()->ReadNext(); break;
                case '>': token = CLpcScanner::Arrow; meme = "->"; File()->ReadNext(); break;
                default : token = CLpcScanner::Minus; meme = "-"; break;
            }
            break;
            
        case '+':
            switch(File()->PeekNext())
            {
                case '+': token = CLpcScanner::PlusPlus; meme = "++"; File()->ReadNext(); break;
                case '=': token = CLpcScanner::PlusEqual; meme = "+="; File()->ReadNext(); break;
                default : token = CLpcScanner::Plus; meme = "+"; break;
            }
            break;
            
        case '*': 
            switch(File()->PeekNext())
            {
                case '=': token = CLpcScanner::TimesEqual; meme  = "*="; File()->ReadNext(); break;
                default : token = CLpcScanner::Star; meme = "*"; break;
            }
            break;

        case '/': 
            switch(File()->PeekNext())
            {
                case '=': token = CLpcScanner::DivideEqual; meme  = "/="; File()->ReadNext(); break;
                default : token = CLpcScanner::Slash; meme = "/"; break;
            }
            break;

        case '%': 
            switch(File()->PeekNext())
            {
                case '=': token = CLpcScanner::ModulusEqual; meme  = "%="; File()->ReadNext(); break;
                default : token = CLpcScanner::Percent; meme = '%'; break;
            }
            break;

        case '|':
            switch(File()->PeekNext())
            {
                case '=': token = CLpcScanner::OrEqual; meme = "|="; File()->ReadNext(); break;
                case '|': token = CLpcScanner::OrOr; meme = "||"; File()->ReadNext(); break;
                default : token = CLpcScanner::Pipe; meme = "|"; break;
            }
            break;

        case '&':
            switch(File()->PeekNext())
            {
                case '=': token = CLpcScanner::AndEqual; meme = "&="; File()->ReadNext(); break;
                case '&': token = CLpcScanner::AndAnd; meme = "&&"; File()->ReadNext(); break;
                default : token = CLpcScanner::Ampersand; meme = "&"; break;
            }
            break;

        case ':':
            switch(File()->PeekNext())
            {
                case ':': token = CLpcScanner::Scope; meme = "::"; File()->ReadNext(); break;
                case ')': token = CLpcScanner::CloseFunction; meme = ":)"; File()->ReadNext(); break;
                default : token = CLpcScanner::Colon; meme = ":"; break;
            }
            break;
            
        case '^':
            switch(File()->PeekNext())
            {
                case '=': token = CLpcScanner::CaratEqual; meme = "^="; File()->ReadNext(); break;
                default : token = CLpcScanner::Carat; meme = "^"; break;
            }
            break;

        case '.':
            if(File()->PeekNext() == '.')
            {
                File()->ReadNext();
                
                if(File()->PeekNext() == '.')
                {
                    token = CLpcScanner::Ellipsis;
                    meme  = "...";
                    File()->ReadNext();
                }
                else
                {
                    token = CLpcScanner::DotDot;
                    meme  = "..";
                }
            }
            else
            {
                token = CLpcScanner::Period;
                meme  = '.';
            }
            break;
            
        case '(':
            switch(File()->PeekNext())
            {
                case ':': token = CLpcScanner::OpenFunction; meme = "(:"; File()->ReadNext(); break;
                case '[': token = CLpcScanner::OpenMapping; meme = "(["; File()->ReadNext(); break;
                case '{': token = CLpcScanner::OpenArray; meme = "({"; File()->ReadNext(); break;
                default : token = CLpcScanner::OpenParen; meme = "("; break;
            }
            break;
        
        case '}':
            switch(File()->PeekNext())
            {
                case ')': token = CLpcScanner::CloseArray; meme = "})";  File()->ReadNext(); break;
                default : token = CLpcScanner::CloseBrace; meme  = "}"; break;
            }
            break;

        case ']':
            if(File()->PeekNext() == ')' && m_inArray == false)
            {
                token = CLpcScanner::CloseMapping;
                meme  = "])";
                File()->ReadNext();
            }
            else
            {
                token     = CLpcScanner::CloseBracket;
                meme      = "]";
                m_inArray = false;
            }
            break;

        case '>':
            if(File()->PeekNext() == '>')
            {
                File()->ReadNext();
                
                if(File()->PeekNext() == '=')
                {
                    token = CLpcScanner::RightShiftEqual;
                    meme  = ">>=";
                    File()->ReadNext();
                }
                else
                {
                    token = CLpcScanner::RightShift;
                    meme  = ">>";
                }
            }
            else if(File()->PeekNext() == '=')
            {
                token = CLpcScanner::GreaterThanEqual;
                meme  = ">=";
                File()->ReadNext();
            }
            else
            {
                token = CLpcScanner::GreaterThan;
                meme  = '>';
            }
            break;
            
        case '<':
            if(File()->PeekNext() == '<')
            {
                File()->ReadNext();
                
                if(File()->PeekNext() == '=')
                {
                    token = CLpcScanner::LeftShiftEqual;
                    meme  = "<<=";
                    File()->ReadNext();
                }
                else
                {
                    token = CLpcScanner::LeftShift;
                    meme  = "<<";
                }
            }
            else if(File()->PeekNext() == '=')
            {
                token = CLpcScanner::LessThanEqual;
                meme  = "<=";
                File()->ReadNext();
            }
            else
            {
                token = CLpcScanner::LessThan;
                meme  = '<';
            }
            break;
            
        case '=':
            switch(File()->PeekNext())
            {
                case '=': token = CLpcScanner::EqualEqual; meme = "=="; File()->ReadNext(); break;
                default : token = CLpcScanner::Assign; meme = "="; break;
            }
            break;

        case '!':
            switch(File()->PeekNext())
            {
                case '=': token = CLpcScanner::NotEqual; meme = "!="; File()->ReadNext(); break;
                default : token = CLpcScanner::Bang; meme = "!"; break;
            }
            break;

        case ',': token = CLpcScanner::Comma; meme = ","; break;
        case ';': token = CLpcScanner::SemiColon; meme = ";"; break;
        case ')': token = CLpcScanner::CloseParen; meme = ")"; break;
        case '{': token = CLpcScanner::OpenBrace; meme = "{"; break;
        case '[': token = CLpcScanner::OpenBracket; meme = "["; m_inArray = true; break;
        case '$': token = CLpcScanner::Dollar; meme = "$"; break;
        case '~': token = CLpcScanner::Tilde; meme = "~"; break;
        case '?': token = CLpcScanner::Question; meme = "?"; break;
    }
        
    if(m_outputStream)
    {
        *m_outputStream << "     >> " << std::setiosflags(std::ios::left);
        *m_outputStream << std::setw(16) << tokenNames[token] << meme << std::endl;
    }

    File()->ReadNext();

    return new CLpcToken(token, meme);
}

// ***************************************************************************
// Function Name: SetOutputStream
//
// Purpose: Set the stream object for debug output
//
// Access: Public
//
// Arguments:
//    None
//
// Return Values:
//    None
//
// Description:
//    If an ostream is passed into this function, all tokenizing is debug
//    spew is sent to it.  Passing in NULL turns this off.  The default value
//    is NULL.  
void 
CLpcScanner::SetOutputStream(std::ostream & stream)
{
    std::stack<CLpcFile *> tempStack;

    m_outputStream = &stream;

    //
    // Set the stream on any and all files in the file stack
    //
    while(!m_fileStack.empty())
    {
        CLpcFile *pFile = m_fileStack.top();

        pFile->SetOutputStream(stream);

        tempStack.push(pFile);
        m_fileStack.pop();
    }

    while(!tempStack.empty())
    {
        m_fileStack.push(tempStack.top());
        tempStack.pop();
    }
}

void 
CLpcScanner::HandleDirective()
{
    std::string directive;
    CHAR        current;

    // Move past the '#'
    current = File()->ReadNext(); 
    
    // Eat any whitespace
    while(File()->EoF() == false && IS_WHITESPACE(current))
    {
        current = File()->ReadNext();
    }

    if(File()->EoF() == true)
    {
        throw ScannerException("End of file in pre-processor directive", 
                               File()->LineNumber(), 
                               File()->Position());
    }

    // Read in the directive type
    while(File()->EoF() == false && IS_LETTER(current))
    {
        directive.append(1, current);
        current = File()->ReadNext();
    }

    if(directive == std::string("include"))
    {
        HandleInclude();
    }
    else if(directive == std::string("define"))
    {
        HandleDefine();
    }
    else if(directive == std::string("if"))
    {
        HandleIf();
    }
    else if(directive == std::string("undef"))
    {
        HandleUndef();
    }
    else if(directive == std::string("ifdef"))
    {
        HandleIfdef();
    }
    else if(directive == std::string("ifndef"))
    {
        HandleIfndef();
    }
    else if(directive == std::string("error"))
    {
        HandleError();
    }
    else if(directive == std::string("warn"))
    {
        HandleWarn();
    }
    else if(directive == std::string("pragma"))
    {
        HandlePragma();
    }
    else
    {
        throw ScannerException("Unknown pre-processor directive", 
                               File()->LineNumber(), 
                               File()->Position());
    }
}

void  
CLpcScanner::HandleInclude()
{
    std::string              fileName;
    std::vector<std::string> lines;
    CHAR                     current;

    current = File()->Current();

    // Eat any whitespace between 'include' and the filename
    while(File()->EoF() == false && IS_WHITESPACE(current))
    {
        current = File()->ReadNext();
    }

    if(File()->EoF() == true)
    {
        throw ScannerException("End of file in #include", 
                               File()->LineNumber(), 
                               File()->Position());
    }

    // Only valid syntax is #include "foo" or #include <foo>
    if(current != '\"' && current != '<')
    {
        throw ScannerException("Invalid #include", 
                               File()->LineNumber(), 
                               File()->Position());
    }

    // Read past the " or <
    current = File()->ReadNext();

    // Read in the file name
    while(File()->EoF() == false && current != '\"' && current != '>')
    {
        fileName.append(1, current);
        current = File()->ReadNext();
    }

    if(File()->EoF() == true)
    {
        throw ScannerException("End of file in #include", 
                               File()->LineNumber(), 
                               File()->Position());
    }

    // TODO:
    // At this point we need to find the file based on include file
    // search rules and using the __INCLUDE_DIRS__ config value
    // but for now we'll just assume it's in the current working dir
    CLpcFile *pFile = new CLpcFile(fileName.c_str());

    pFile->SetOutputStream(*m_outputStream);
    pFile->Load();

    // Make the included file the top of the stack
    m_fileStack.push(pFile);

    //
    // Read to the first non-whitespace character of the new file
    //
    current = File()->ReadNext();

    while(File()->EoF() == false && IS_WHITESPACE(current))
    {
        current = File()->ReadNext();
    }
}

void  
CLpcScanner::HandleDefine()
{
    std::string ident;
    CHAR        current;

    current = File()->Current();

    // Eat any whitespace between 'include' and the filename
    while(File()->EoF() == false && IS_WHITESPACE(current))
    {
        current = File()->ReadNext();
    }

    if(File()->EoF() == true)
    {
        throw ScannerException("End of file in #define", 
                               File()->LineNumber(), 
                               File()->Position());
    }
    
    if(!IS_LETTER(current) && current != '_')
    {
        throw ScannerException("Invalid #define identifier", 
                               File()->LineNumber(), 
                               File()->Position());
    }
    
    while(File()->EoF() == false && !IS_WHITESPACE(current))
    {
        ident.append(1, current);
        current = File()->ReadNext();
    }
    
    if(File()->EoF() == true)
    {
        throw ScannerException("End of file in #define", 
                               File()->LineNumber(), 
                               File()->Position());
    }
    
    if(current == '(')
    {
        // It's a function style macro
    }
    else
    {
        INT         line;
        std::string value;
        bool        done;
        CHAR        last;
        
        // It's a simple replacement macro
        done = false;
        line = File()->LineNumber();
        last = current;
        
        while(done == false)
        {
            // Read characters until the end of the line
            while(File()->EoF() == false && line == File()->LineNumber())
            {
                value.append(1, current);
                
                last    = current;
                current = File()->ReadNext();
            }
            
            if(File()->EoF() == true)
            {
                throw ScannerException("End of file in #define", 
                                       File()->LineNumber(), 
                                       File()->Position());
            }
            
            // If the last character on the line is the backslash
            // the macro continues on the next line
            if(last == '\\')
            {
                value.erase(value.end());
                
                line    = File()->LineNumber();
                current = File()->ReadNext();
            }
            else
            {
                done = true;
            }
        }
    }
}

void  
CLpcScanner::HandleIf()
{
}

void  
CLpcScanner::HandleUndef()
{
}

void  
CLpcScanner::HandleIfdef()
{
}

void  
CLpcScanner::HandleIfndef()
{
}

void  
CLpcScanner::HandleError()
{
}

void  
CLpcScanner::HandleWarn()
{
}

void  
CLpcScanner::HandlePragma()
{
}

//
// Definitions for CLpcFile
//

CLpcFile::CLpcFile(std::string fileName)
    : m_fileName(fileName),
      m_currentLine(""),
      m_outputStream(0),
      m_currentChar(' '),
      m_currentIndex(0),
      m_lineNumber(0),
      m_eof(false),
      m_inString(false)
{
}

CLpcFile::~CLpcFile()
{
    if(m_outputStream)
    {
        *m_outputStream << "<<< " << m_fileName << std::endl << std::endl;
    }
}

// ***************************************************************************
// Function Name: Load
//
// Purpose: Load input file
//
// Access: Private
//
// Arguments:
//    None
//
// Return Values:
//    void
//
// Description:
//    Reads in the input file line by line adding each line as a std::string
//    to a std::vector.  The result is the m_lines member variable.
void
CLpcFile::Load()
{
    std::ifstream fd(m_fileName.c_str());

    while(fd.good())
    {
        std::string line;

        // Pull a line from the file
        getline(fd, line);

        // Ignore empty lines or comments
        if(line.length() == 0 || line[0] == '\n')
        {
            continue;
        }

        m_lines.push_back(line);
    }

    fd.close();

    m_lineIterator = m_lines.begin();

    if(m_lineIterator == m_lines.end())
    {
        // Empty file
        m_eof = true;
        return;
    }

    m_currentLine   = *m_lineIterator;
    m_lineNumber    = 1;
    m_currentIndex  = 0;

    if(m_outputStream)
    {
        *m_outputStream << std::endl << ">>> " << m_fileName << std::endl;
        *m_outputStream << std::setw(4) << std::setiosflags(std::ios::left);
        *m_outputStream << m_lineNumber << ": " << m_currentLine << std::endl;
    }
}

// ***************************************************************************
// Function Name: ReadNext
//
// Purpose: Scan the next character in the input
//
// Access: Private
//
// Arguments:
//    None
//
// Return Values:
//    CHAR - The new value read
//
// Description:
//    Whenever this is called the next character in the input will be loaded
//    into m_currentChar.  If the end of a line is reached, the next line of
//    input is loaded into m_currentLine.  At the end of the file m_eof is set
//    to true.
CHAR
CLpcFile::ReadNext()
{
    bool haveChar  = false;
    bool inComment = false;

    do
    {
        if(m_currentIndex == m_currentLine.length())
        {
            //
            // Finished with the current line, get the next
            //
            m_lineIterator++;

            if(m_lineIterator == m_lines.end())
            {
                // End of File
                m_eof = true;

                if(inComment == true)
                {
                    throw ScannerException("End of file in comment", m_lineNumber, m_currentIndex);
                }

                return ' ';
            }

            m_currentLine   = *m_lineIterator;
            m_currentIndex  = 0;
            m_lineNumber   += 1;

            if(m_outputStream)
            {
                *m_outputStream << std::setw(4) << std::setiosflags(std::ios::left);
                *m_outputStream << m_lineNumber << ": " << m_currentLine << std::endl;
            }
        }

        // Read the next character
        m_currentChar = m_currentLine[m_currentIndex++];

        //
        // Check for comments
        //
        if(inComment == true)
        {
            if(m_currentChar == '*' && PeekNext() == '/')
            {
                inComment       = false;
                m_currentIndex += 1;
            }

            continue;
        }

        if(m_currentChar == '/' && m_inString == false)
        {
            switch(PeekNext())
            {
                case '/':
                    // C++ style comment, so just skip to the next line
                    m_currentIndex = m_currentLine.length();
                    continue;

                case '*':
                    // C style comment, set a flag and start skipping characters
                    inComment       = true;
                    m_currentIndex += 1;
                    continue;
            }
        }

        haveChar = true;
    }
    while(haveChar == false);

    return m_currentChar;
}

// ***************************************************************************
// Function Name: PeekNextChar
//
// Purpose: Get the next character in the input
//
// Access: Private
//
// Arguments:
//    None
//
// Return Values:
//    CHAR - The next value in the input
//
// Description:
//    This function will return the character after m_currentChar on the
//    m_currentLine.  If there are not more characters in the line, 0 is
//    returned.
CHAR
CLpcFile::PeekNext()
{
    if(m_eof == true || m_currentIndex == m_currentLine.length())
    {
        // If we're at the end of the line or file, return 0
        return 0;
    }

    return m_currentLine[m_currentIndex];
}

// ***************************************************************************
// Function Name: SetOutputStream
//
// Purpose: Set the stream object for debug output
//
// Access: Public
//
// Arguments:
//    None
//
// Return Values:
//    None
//
// Description:
//    If an ostream is passed into this function, all tokenizing is debug
//    spew is sent to it.  Passing in NULL turns this off.  The default value
//    is NULL.
void 
CLpcFile::SetOutputStream(std::ostream & stream)
{
    m_outputStream = &stream;
}
